{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "source": [
    "import torch\n",
    "import torch.nn as nn"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：input_size(输入张量x的维度)\n",
    "第二个参数：hidden_size(隐藏层的维度， 隐藏层的神经元个数)\n",
    "第三个参数：num_layer(隐藏层的数量)\n",
    "'''\n",
    "rnn = nn.RNN(5, 6, 1)  # --A"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：sequence_length(输入序列的长度)\n",
    "第二个参数：batch_size(批次的样本数量)\n",
    "第三个参数：input_size(输入张量的维度)\n",
    "'''\n",
    "input = torch.randn(1, 3, 5) #--B"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：num_layer * num_directions(层数*网络方向)\n",
    "第二个参数：batch_size(批次的样本数)\n",
    "第三个参数：hidden_size(隐藏层的维度， 隐藏层神经元的个数)\n",
    "'''\n",
    "h0 = torch.randn(1, 3, 6) #--C"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "source": [
    "# 注意点： 上面中的三个一的意义不一样 A和C的一需要保持一致， 也可以自己指定 \n",
    "# B中的1可以自己任意改写， 因为是程序员自己指定，"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "source": [
    "# 输入input到RNN中，得到结果\n",
    "output, hn = rnn(input, h0)\n",
    "\n",
    "print(output)\n",
    "print(output.shape)\n",
    "print(hn)\n",
    "print(hn.shape)"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：input_size(输入张量x的维度)\n",
    "第二个参数：hidden_size(隐藏层的维度， 隐藏层的神经元个数)\n",
    "第三个参数：num_layer(隐藏层层数)\n",
    "'''\n",
    "lstm = nn.LSTM(5, 6, 2)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：sequence_length(输入序列的长度)\n",
    "第二个参数：batch_size(批次的样本数量)\n",
    "第三个参数：input_size(输入张量x的维度)\n",
    "'''\n",
    "input1 = torch.randn(1, 3, 5)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：num_layer * num_directions(隐藏层层数*方向数)\n",
    "第二个参数：batch_size(批次的样本数量)\n",
    "第三个参数：num_layer(隐藏层的维度)\n",
    "'''\n",
    "h0 = torch.randn(2, 3, 6)\n",
    "c0 = torch.randn(2, 3, 6)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "source": [
    "# 将input1,  h0, c0输入到lstm中， 输出结果\n",
    "output, (hn, cn) = lstm(input1, (h0, c0))\n",
    "\n",
    "print(output)\n",
    "print(output.shape)\n",
    "print(hn)\n",
    "print(hn.shape)\n",
    "print(cn)\n",
    "print(cn.shape)"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GRU模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：input_size(输入张量x的维度)\n",
    "第二个参数：hidden_size(隐藏层的维度， 隐藏层神经元的个数)\n",
    "第三个参数：num_layers(隐藏层的层数)\n",
    "'''\n",
    "gru = nn.GRU(5, 6, 2)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：sequence_length(输入序列的长度)\n",
    "第二个参数：batch_siz(批次样本的数量)\n",
    "第三个参数：input_size(输入张量x的维度)\n",
    "'''\n",
    "input2 = torch.randn(1, 3, 5)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "source": [
    "'''\n",
    "第一个参数：num_layers * num_directions(隐藏层的层数 * 方向数)\n",
    "第二个参数：batch_size(批次样本的数量)\n",
    "第三个参数：hidden_size(隐藏层的维度， 隐藏层的神经元个数)\n",
    "'''\n",
    "h0 = torch.randn(2, 3, 6)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "source": [
    "output, hn = gru(input2, h0)\n",
    "print(output)\n",
    "print(output.shape)\n",
    "print(hn)\n",
    "print(hn.shape)"
   ],
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 注意力机制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "source": [
    "# query： 查询   key:  关键字  value：值\n",
    "# 注意力机制各个的含义：https://blog.csdn.net/ningyanggege/article/details/89786077"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "source": [
    "class Attn(nn.Module):\n",
    "    def __init__(self, query_size, key_size, value_size1, value_size2, output_size):\n",
    "        \"\"\"初始化函数中的参数有5个, query_size代表query的最后一维大小\n",
    "           key_size代表key的最后一维大小, value_size1代表value的导数第二维大小, \n",
    "           value = (1, value_size1, value_size2)\n",
    "           value_size2代表value的倒数第一维大小, output_size输出的最后一维大小\"\"\"\n",
    "        super(Attn, self).__init__()\n",
    "        # 将以下参数传入类中\n",
    "        self.query_size = query_size\n",
    "        self.key_size = key_size\n",
    "        self.value_size1 = value_size1\n",
    "        self.value_size2 = value_size2\n",
    "        self.output_size = output_size\n",
    "\n",
    "        # 初始化注意力机制实现第一步中需要的线性层.\n",
    "        self.attn = nn.Linear(self.query_size + self.key_size, value_size1)\n",
    "\n",
    "        # 初始化注意力机制实现第三步中需要的线性层.\n",
    "        self.attn_combine = nn.Linear(self.query_size + value_size2, output_size)\n",
    "\n",
    "\n",
    "    def forward(self, Q, K, V):\n",
    "        \"\"\"forward函数的输入参数有三个, 分别是Q, K, V, 根据模型训练常识, 输入给Attion机制的\n",
    "           张量一般情况都是三维张量, 因此这里也假设Q, K, V都是三维张量\"\"\"\n",
    "\n",
    "        # 第一步, 按照计算规则进行计算, \n",
    "        # 我们采用常见的第一种计算规则\n",
    "        # 将Q，K进行纵轴拼接, 做一次线性变化, 最后使用softmax处理获得结果\n",
    "        attn_weights = F.softmax(\n",
    "            self.attn(torch.cat((Q[0], K[0]), 1)), dim=1)\n",
    "\n",
    "        # 然后进行第一步的后半部分, 将得到的权重矩阵与V做矩阵乘法计算, \n",
    "        # 当二者都是三维张量且第一维代表为batch条数时, 则做bmm运算\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0), V)\n",
    "\n",
    "        # 之后进行第二步, 通过取[0]是用来降维, 根据第一步采用的计算方法, \n",
    "        # 需要将Q与第一步的计算结果再进行拼接\n",
    "        output = torch.cat((Q[0], attn_applied[0]), 1)\n",
    "\n",
    "        # 最后是第三步, 使用线性层作用在第三步的结果上做一个线性变换并扩展维度，得到输出\n",
    "        # 因为要保证输出也是三维张量, 因此使用unsqueeze(0)扩展维度\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "        return output, attn_weights"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "source": [
    "query_size = 32\n",
    "key_size = 32\n",
    "value_size1 = 32\n",
    "value_size2 = 64\n",
    "output_size = 64\n",
    "\n",
    "attn = Attn(query_size, key_size, value_size1, value_size2, output_size)\n",
    "\n",
    "Q = torch.randn(1, 1, 32)\n",
    "K = torch.randn(1, 1, 32)\n",
    "V = torch.randn(1, 32, 64)"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "source": [
    "Q"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "source": [
    "Q[0]"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "source": [
    "K"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "source": [
    "K[0]"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "source": [
    "V"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "source": [
    "out = attn(Q, K, V)\n",
    "print(out[0])\n",
    "print(out[1])"
   ],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "source": [],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "source": [],
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "source": [],
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
